{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\ProgramData\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From d:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n",
      "WARNING:tensorflow:From <ipython-input-3-f958ba5ec233>:22: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From d:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From d:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting data1\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From d:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting data1\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From d:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting data1\\t10k-images-idx3-ubyte.gz\n",
      "Extracting data1\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From d:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np  # 矩阵操作\n",
    "from numpy import array\n",
    "\n",
    "FLAGS = None\n",
    "\n",
    "# Import data\n",
    "data_dir = 'data1'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "train image size= 55000\n",
      "train label size= 55000\n",
      "test image size= 10000\n",
      "test label size= 10000\n",
      "feature size= 784\n",
      "class size= 10\n"
     ]
    }
   ],
   "source": [
    "print(type(mnist.train.images))\n",
    "print(type(mnist.train.labels))\n",
    "print(type(mnist.test.images))\n",
    "print(type(mnist.test.labels))\n",
    "\n",
    "print(\"train image size=\",len(mnist.train.images))\n",
    "print(\"train label size=\",len(mnist.train.labels))\n",
    "print(\"test image size=\",len(mnist.test.images))\n",
    "print(\"test label size=\",len(mnist.test.labels))\n",
    "\n",
    "print(\"feature size=\",len(mnist.train.images[0]))\n",
    "print(\"class size=\",len(mnist.train.labels[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = tf.placeholder(tf.float32)\n",
    "# Layer1\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W_1 = tf.Variable(tf.random_normal([784, 392], stddev=0.05))\n",
    "b_1 = tf.Variable(tf.random_normal([392], stddev=0.05))\n",
    "y_1 = tf.matmul(x, W_1) + b_1\n",
    "h_1 = tf.nn.sigmoid(y_1)\n",
    "# Layer 2\n",
    "W_2 = tf.Variable(tf.random_normal([392, 100], stddev=0.05))\n",
    "b_2 = tf.Variable(tf.random_normal([100], stddev=0.05))\n",
    "y_2 = tf.matmul(h_1, W_2) + b_2\n",
    "h_2 = tf.nn.sigmoid(y_2)\n",
    "# Layer 3\n",
    "W_3 = tf.Variable(tf.random_normal([100, 10], stddev=0.06))\n",
    "b_3 = tf.Variable(tf.random_normal([10], stddev=0.06))\n",
    "y = tf.matmul(h_2, W_3) + b_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=y))\n",
    "\n",
    "l2_loss = tf.nn.l2_loss(W_1) + tf.nn.l2_loss(W_2) + tf.nn.l2_loss(W_3)\n",
    "total_loss = cross_entropy + l2_loss * 7e-5\n",
    "\n",
    "#train_step = tf.train.AdamOptimizer(0.5).minimize(cross_entropy)\n",
    "#train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "train_step = tf.train.AdamOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "# Test trained model\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, lr: 0.001000, entropy loss: 0.950876, l2 loss: 499.125854, total loss: 0.985815\n",
      "train accuracy: 0.760000\n",
      "test accuracy: 0.785100\n",
      "\n",
      "step 200, lr: 0.001000, entropy loss: 0.474404, l2 loss: 637.498230, total loss: 0.519029\n",
      "train accuracy: 0.900000\n",
      "test accuracy: 0.887400\n",
      "\n",
      "step 300, lr: 0.001000, entropy loss: 0.407207, l2 loss: 705.598816, total loss: 0.456599\n",
      "train accuracy: 0.910000\n",
      "test accuracy: 0.905300\n",
      "\n",
      "step 400, lr: 0.001000, entropy loss: 0.344598, l2 loss: 735.361450, total loss: 0.396073\n",
      "train accuracy: 0.880000\n",
      "test accuracy: 0.914100\n",
      "\n",
      "step 500, lr: 0.001000, entropy loss: 0.436487, l2 loss: 751.362854, total loss: 0.489082\n",
      "train accuracy: 0.870000\n",
      "test accuracy: 0.918300\n",
      "\n",
      "step 600, lr: 0.001000, entropy loss: 0.282257, l2 loss: 766.442627, total loss: 0.335908\n",
      "train accuracy: 0.880000\n",
      "test accuracy: 0.925200\n",
      "\n",
      "step 700, lr: 0.000950, entropy loss: 0.321951, l2 loss: 777.603699, total loss: 0.376383\n",
      "train accuracy: 0.920000\n",
      "test accuracy: 0.932000\n",
      "\n",
      "step 800, lr: 0.000950, entropy loss: 0.181811, l2 loss: 790.329102, total loss: 0.237134\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.934500\n",
      "\n",
      "step 900, lr: 0.000950, entropy loss: 0.156260, l2 loss: 800.150818, total loss: 0.212271\n",
      "train accuracy: 0.950000\n",
      "test accuracy: 0.934400\n",
      "\n",
      "step 1000, lr: 0.000950, entropy loss: 0.126382, l2 loss: 807.622070, total loss: 0.182916\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.940800\n",
      "\n",
      "step 1100, lr: 0.000950, entropy loss: 0.241404, l2 loss: 816.220825, total loss: 0.298539\n",
      "train accuracy: 0.910000\n",
      "test accuracy: 0.941800\n",
      "\n",
      "step 1200, lr: 0.000950, entropy loss: 0.089519, l2 loss: 824.360718, total loss: 0.147224\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.945900\n",
      "\n",
      "step 1300, lr: 0.000902, entropy loss: 0.162950, l2 loss: 828.895020, total loss: 0.220972\n",
      "train accuracy: 0.960000\n",
      "test accuracy: 0.942400\n",
      "\n",
      "step 1400, lr: 0.000902, entropy loss: 0.118532, l2 loss: 837.144714, total loss: 0.177132\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.948100\n",
      "\n",
      "step 1500, lr: 0.000902, entropy loss: 0.252840, l2 loss: 845.957642, total loss: 0.312057\n",
      "train accuracy: 0.920000\n",
      "test accuracy: 0.949000\n",
      "\n",
      "step 1600, lr: 0.000902, entropy loss: 0.119838, l2 loss: 844.154907, total loss: 0.178929\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.951600\n",
      "\n",
      "step 1700, lr: 0.000902, entropy loss: 0.175587, l2 loss: 849.981445, total loss: 0.235086\n",
      "train accuracy: 0.930000\n",
      "test accuracy: 0.952000\n",
      "\n",
      "step 1800, lr: 0.000902, entropy loss: 0.127413, l2 loss: 854.876465, total loss: 0.187254\n",
      "train accuracy: 0.960000\n",
      "test accuracy: 0.954600\n",
      "\n",
      "step 1900, lr: 0.000857, entropy loss: 0.145129, l2 loss: 860.136536, total loss: 0.205338\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.955400\n",
      "\n",
      "step 2000, lr: 0.000857, entropy loss: 0.119325, l2 loss: 865.934448, total loss: 0.179940\n",
      "train accuracy: 0.960000\n",
      "test accuracy: 0.959400\n",
      "\n",
      "step 2100, lr: 0.000857, entropy loss: 0.109052, l2 loss: 866.919922, total loss: 0.169736\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.957600\n",
      "\n",
      "step 2200, lr: 0.000857, entropy loss: 0.179937, l2 loss: 871.454102, total loss: 0.240939\n",
      "train accuracy: 0.950000\n",
      "test accuracy: 0.962300\n",
      "\n",
      "step 2300, lr: 0.000857, entropy loss: 0.163009, l2 loss: 874.338562, total loss: 0.224213\n",
      "train accuracy: 0.950000\n",
      "test accuracy: 0.961800\n",
      "\n",
      "step 2400, lr: 0.000857, entropy loss: 0.112951, l2 loss: 875.362610, total loss: 0.174226\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.964100\n",
      "\n",
      "step 2500, lr: 0.000815, entropy loss: 0.172963, l2 loss: 877.592346, total loss: 0.234395\n",
      "train accuracy: 0.960000\n",
      "test accuracy: 0.961400\n",
      "\n",
      "step 2600, lr: 0.000815, entropy loss: 0.163197, l2 loss: 882.823547, total loss: 0.224994\n",
      "train accuracy: 0.960000\n",
      "test accuracy: 0.961900\n",
      "\n",
      "step 2700, lr: 0.000815, entropy loss: 0.131231, l2 loss: 884.212646, total loss: 0.193126\n",
      "train accuracy: 0.940000\n",
      "test accuracy: 0.964600\n",
      "\n",
      "step 2800, lr: 0.000815, entropy loss: 0.118988, l2 loss: 885.336914, total loss: 0.180962\n",
      "train accuracy: 0.950000\n",
      "test accuracy: 0.964900\n",
      "\n",
      "step 2900, lr: 0.000815, entropy loss: 0.112855, l2 loss: 888.014587, total loss: 0.175016\n",
      "train accuracy: 0.960000\n",
      "test accuracy: 0.965900\n",
      "\n",
      "step 3000, lr: 0.000815, entropy loss: 0.124594, l2 loss: 891.209778, total loss: 0.186979\n",
      "train accuracy: 0.960000\n",
      "test accuracy: 0.967300\n",
      "\n",
      "step 3100, lr: 0.000774, entropy loss: 0.087516, l2 loss: 894.531006, total loss: 0.150133\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.968800\n",
      "\n",
      "step 3200, lr: 0.000774, entropy loss: 0.040788, l2 loss: 892.040833, total loss: 0.103231\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.967800\n",
      "\n",
      "step 3300, lr: 0.000774, entropy loss: 0.103926, l2 loss: 894.228760, total loss: 0.166522\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.967000\n",
      "\n",
      "step 3400, lr: 0.000774, entropy loss: 0.077575, l2 loss: 892.685547, total loss: 0.140063\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.968000\n",
      "\n",
      "step 3500, lr: 0.000774, entropy loss: 0.086653, l2 loss: 894.098267, total loss: 0.149240\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.971100\n",
      "\n",
      "step 3600, lr: 0.000774, entropy loss: 0.084302, l2 loss: 893.024353, total loss: 0.146813\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.968100\n",
      "\n",
      "step 3700, lr: 0.000735, entropy loss: 0.036511, l2 loss: 894.834839, total loss: 0.099149\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.969300\n",
      "\n",
      "step 3800, lr: 0.000735, entropy loss: 0.101710, l2 loss: 895.574890, total loss: 0.164401\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.971800\n",
      "\n",
      "step 3900, lr: 0.000735, entropy loss: 0.106094, l2 loss: 897.482788, total loss: 0.168917\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.971600\n",
      "\n",
      "step 4000, lr: 0.000735, entropy loss: 0.170417, l2 loss: 898.355652, total loss: 0.233301\n",
      "train accuracy: 0.930000\n",
      "test accuracy: 0.970700\n",
      "\n",
      "step 4100, lr: 0.000735, entropy loss: 0.070754, l2 loss: 899.637390, total loss: 0.133729\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.970800\n",
      "\n",
      "step 4200, lr: 0.000735, entropy loss: 0.031703, l2 loss: 902.107239, total loss: 0.094851\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.972000\n",
      "\n",
      "step 4300, lr: 0.000698, entropy loss: 0.071639, l2 loss: 901.154846, total loss: 0.134719\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.972200\n",
      "\n",
      "step 4400, lr: 0.000698, entropy loss: 0.099457, l2 loss: 902.216187, total loss: 0.162612\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.971500\n",
      "\n",
      "step 4500, lr: 0.000698, entropy loss: 0.033336, l2 loss: 901.817261, total loss: 0.096463\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.973400\n",
      "\n",
      "step 4600, lr: 0.000698, entropy loss: 0.068534, l2 loss: 898.433350, total loss: 0.131424\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.974100\n",
      "\n",
      "step 4700, lr: 0.000698, entropy loss: 0.079086, l2 loss: 900.227722, total loss: 0.142102\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.973700\n",
      "\n",
      "step 4800, lr: 0.000698, entropy loss: 0.027904, l2 loss: 901.001343, total loss: 0.090974\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.972900\n",
      "\n",
      "step 4900, lr: 0.000663, entropy loss: 0.094171, l2 loss: 902.406738, total loss: 0.157339\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.974300\n",
      "\n",
      "step 5000, lr: 0.000663, entropy loss: 0.039818, l2 loss: 902.791260, total loss: 0.103014\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.975600\n",
      "\n",
      "step 5100, lr: 0.000663, entropy loss: 0.046013, l2 loss: 902.129272, total loss: 0.109162\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.973800\n",
      "\n",
      "step 5200, lr: 0.000663, entropy loss: 0.047921, l2 loss: 901.654175, total loss: 0.111036\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.974700\n",
      "\n",
      "step 5300, lr: 0.000663, entropy loss: 0.079165, l2 loss: 900.954956, total loss: 0.142232\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.975600\n",
      "\n",
      "step 5400, lr: 0.000663, entropy loss: 0.091195, l2 loss: 900.887451, total loss: 0.154257\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.975400\n",
      "\n",
      "step 5500, lr: 0.000630, entropy loss: 0.093534, l2 loss: 902.456421, total loss: 0.156706\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.975200\n",
      "\n",
      "step 5600, lr: 0.000630, entropy loss: 0.062109, l2 loss: 903.331299, total loss: 0.125342\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.973600\n",
      "\n",
      "step 5700, lr: 0.000630, entropy loss: 0.015988, l2 loss: 902.241516, total loss: 0.079145\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.975900\n",
      "\n",
      "step 5800, lr: 0.000630, entropy loss: 0.077902, l2 loss: 902.014160, total loss: 0.141043\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.974500\n",
      "\n",
      "step 5900, lr: 0.000630, entropy loss: 0.064832, l2 loss: 902.366089, total loss: 0.127997\n",
      "train accuracy: 0.970000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test accuracy: 0.975600\n",
      "\n",
      "step 6000, lr: 0.000630, entropy loss: 0.099378, l2 loss: 902.703491, total loss: 0.162567\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.975700\n",
      "\n",
      "step 6100, lr: 0.000599, entropy loss: 0.045574, l2 loss: 902.725464, total loss: 0.108765\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.974600\n",
      "\n",
      "step 6200, lr: 0.000599, entropy loss: 0.032538, l2 loss: 903.039856, total loss: 0.095751\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.975900\n",
      "\n",
      "step 6300, lr: 0.000599, entropy loss: 0.019231, l2 loss: 902.642761, total loss: 0.082416\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.974200\n",
      "\n",
      "step 6400, lr: 0.000599, entropy loss: 0.068688, l2 loss: 902.373413, total loss: 0.131854\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.977100\n",
      "\n",
      "step 6500, lr: 0.000599, entropy loss: 0.039581, l2 loss: 902.152222, total loss: 0.102732\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.975800\n",
      "\n",
      "step 6600, lr: 0.000599, entropy loss: 0.024184, l2 loss: 901.104004, total loss: 0.087262\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.976700\n",
      "\n",
      "step 6700, lr: 0.000569, entropy loss: 0.047369, l2 loss: 899.822876, total loss: 0.110357\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.976900\n",
      "\n",
      "step 6800, lr: 0.000569, entropy loss: 0.066669, l2 loss: 900.622803, total loss: 0.129712\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.976900\n",
      "\n",
      "step 6900, lr: 0.000569, entropy loss: 0.033217, l2 loss: 898.966248, total loss: 0.096145\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.976700\n",
      "\n",
      "step 7000, lr: 0.000569, entropy loss: 0.075507, l2 loss: 898.740051, total loss: 0.138419\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.974700\n",
      "\n",
      "step 7100, lr: 0.000569, entropy loss: 0.061574, l2 loss: 900.654846, total loss: 0.124619\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.973600\n",
      "\n",
      "step 7200, lr: 0.000569, entropy loss: 0.038628, l2 loss: 899.502319, total loss: 0.101593\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.974500\n",
      "\n",
      "step 7300, lr: 0.000540, entropy loss: 0.033009, l2 loss: 899.429688, total loss: 0.095969\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.976400\n",
      "\n",
      "step 7400, lr: 0.000540, entropy loss: 0.053724, l2 loss: 900.026917, total loss: 0.116726\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.977500\n",
      "\n",
      "step 7500, lr: 0.000540, entropy loss: 0.046590, l2 loss: 900.766174, total loss: 0.109643\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.977700\n",
      "\n",
      "step 7600, lr: 0.000540, entropy loss: 0.107770, l2 loss: 900.372437, total loss: 0.170796\n",
      "train accuracy: 0.950000\n",
      "test accuracy: 0.977500\n",
      "\n",
      "step 7700, lr: 0.000540, entropy loss: 0.027158, l2 loss: 900.039978, total loss: 0.090161\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.976600\n",
      "\n",
      "step 7800, lr: 0.000540, entropy loss: 0.050081, l2 loss: 898.381958, total loss: 0.112968\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.977800\n",
      "\n",
      "step 7900, lr: 0.000513, entropy loss: 0.078073, l2 loss: 898.523499, total loss: 0.140970\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.978200\n",
      "\n",
      "step 8000, lr: 0.000513, entropy loss: 0.035802, l2 loss: 897.291016, total loss: 0.098612\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.977200\n",
      "\n",
      "step 8100, lr: 0.000513, entropy loss: 0.013973, l2 loss: 895.965088, total loss: 0.076690\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.977900\n",
      "\n",
      "step 8200, lr: 0.000513, entropy loss: 0.043941, l2 loss: 895.751465, total loss: 0.106644\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.976900\n",
      "\n",
      "step 8300, lr: 0.000513, entropy loss: 0.055579, l2 loss: 896.800598, total loss: 0.118355\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.979800\n",
      "\n",
      "step 8400, lr: 0.000513, entropy loss: 0.019969, l2 loss: 896.985535, total loss: 0.082758\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.977300\n",
      "\n",
      "step 8500, lr: 0.000488, entropy loss: 0.060537, l2 loss: 895.340698, total loss: 0.123211\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.979100\n",
      "\n",
      "step 8600, lr: 0.000488, entropy loss: 0.090003, l2 loss: 894.449341, total loss: 0.152614\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.976900\n",
      "\n",
      "step 8700, lr: 0.000488, entropy loss: 0.031749, l2 loss: 894.143799, total loss: 0.094339\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.977900\n",
      "\n",
      "step 8800, lr: 0.000488, entropy loss: 0.013536, l2 loss: 893.063538, total loss: 0.076050\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.978200\n",
      "\n",
      "step 8900, lr: 0.000488, entropy loss: 0.053115, l2 loss: 894.893921, total loss: 0.115758\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.977700\n",
      "\n",
      "step 9000, lr: 0.000488, entropy loss: 0.023791, l2 loss: 893.523743, total loss: 0.086337\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.978100\n",
      "\n",
      "step 9100, lr: 0.000463, entropy loss: 0.034480, l2 loss: 893.649414, total loss: 0.097036\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.976900\n",
      "\n",
      "step 9200, lr: 0.000463, entropy loss: 0.020350, l2 loss: 892.561157, total loss: 0.082829\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.978500\n",
      "\n",
      "step 9300, lr: 0.000463, entropy loss: 0.047113, l2 loss: 892.752808, total loss: 0.109605\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.977900\n",
      "\n",
      "step 9400, lr: 0.000463, entropy loss: 0.043873, l2 loss: 892.515137, total loss: 0.106349\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.979400\n",
      "\n",
      "step 9500, lr: 0.000463, entropy loss: 0.050260, l2 loss: 891.730957, total loss: 0.112682\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.979500\n",
      "\n",
      "step 9600, lr: 0.000463, entropy loss: 0.127635, l2 loss: 890.076355, total loss: 0.189940\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.978500\n",
      "\n",
      "step 9700, lr: 0.000440, entropy loss: 0.024463, l2 loss: 890.532715, total loss: 0.086800\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.978300\n",
      "\n",
      "step 9800, lr: 0.000440, entropy loss: 0.025634, l2 loss: 890.043152, total loss: 0.087937\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.978700\n",
      "\n",
      "step 9900, lr: 0.000440, entropy loss: 0.033726, l2 loss: 889.844727, total loss: 0.096015\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.976700\n",
      "\n",
      "step 10000, lr: 0.000440, entropy loss: 0.039739, l2 loss: 890.351196, total loss: 0.102064\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.979400\n",
      "\n",
      "step 10100, lr: 0.000440, entropy loss: 0.032332, l2 loss: 889.195679, total loss: 0.094576\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.977600\n",
      "\n",
      "step 10200, lr: 0.000440, entropy loss: 0.066638, l2 loss: 889.757324, total loss: 0.128921\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.978200\n",
      "\n",
      "step 10300, lr: 0.000418, entropy loss: 0.035441, l2 loss: 889.221313, total loss: 0.097687\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.979200\n",
      "\n",
      "step 10400, lr: 0.000418, entropy loss: 0.084983, l2 loss: 889.014709, total loss: 0.147214\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.979700\n",
      "\n",
      "step 10500, lr: 0.000418, entropy loss: 0.037719, l2 loss: 887.617004, total loss: 0.099853\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.979200\n",
      "\n",
      "step 10600, lr: 0.000418, entropy loss: 0.030169, l2 loss: 887.383301, total loss: 0.092286\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.978500\n",
      "\n",
      "step 10700, lr: 0.000418, entropy loss: 0.028340, l2 loss: 886.632019, total loss: 0.090404\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.979400\n",
      "\n",
      "step 10800, lr: 0.000418, entropy loss: 0.042963, l2 loss: 885.741455, total loss: 0.104965\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.978700\n",
      "\n",
      "step 10900, lr: 0.000397, entropy loss: 0.037053, l2 loss: 885.321594, total loss: 0.099025\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.978700\n",
      "\n",
      "step 11000, lr: 0.000397, entropy loss: 0.045684, l2 loss: 884.445007, total loss: 0.107595\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.980000\n",
      "\n",
      "step 11100, lr: 0.000397, entropy loss: 0.063120, l2 loss: 884.072937, total loss: 0.125006\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.980600\n",
      "\n",
      "step 11200, lr: 0.000397, entropy loss: 0.055005, l2 loss: 882.799561, total loss: 0.116801\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.979900\n",
      "\n",
      "step 11300, lr: 0.000397, entropy loss: 0.043125, l2 loss: 883.154724, total loss: 0.104946\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.978900\n",
      "\n",
      "step 11400, lr: 0.000397, entropy loss: 0.025272, l2 loss: 882.998413, total loss: 0.087082\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.978600\n",
      "\n",
      "step 11500, lr: 0.000377, entropy loss: 0.076502, l2 loss: 882.256836, total loss: 0.138260\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.978900\n",
      "\n",
      "step 11600, lr: 0.000377, entropy loss: 0.032572, l2 loss: 881.247009, total loss: 0.094259\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.979500\n",
      "\n",
      "step 11700, lr: 0.000377, entropy loss: 0.015854, l2 loss: 881.043945, total loss: 0.077527\n",
      "train accuracy: 1.000000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test accuracy: 0.979800\n",
      "\n",
      "step 11800, lr: 0.000377, entropy loss: 0.017691, l2 loss: 880.140564, total loss: 0.079300\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.979700\n",
      "\n",
      "step 11900, lr: 0.000377, entropy loss: 0.040327, l2 loss: 879.422485, total loss: 0.101886\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.980500\n",
      "\n",
      "step 12000, lr: 0.000377, entropy loss: 0.013716, l2 loss: 879.372620, total loss: 0.075272\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.979200\n",
      "\n",
      "step 12100, lr: 0.000358, entropy loss: 0.023004, l2 loss: 878.810059, total loss: 0.084521\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981400\n",
      "\n",
      "step 12200, lr: 0.000358, entropy loss: 0.017623, l2 loss: 878.886597, total loss: 0.079145\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980500\n",
      "\n",
      "step 12300, lr: 0.000358, entropy loss: 0.046192, l2 loss: 878.022583, total loss: 0.107654\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.980400\n",
      "\n",
      "step 12400, lr: 0.000358, entropy loss: 0.032642, l2 loss: 878.171387, total loss: 0.094114\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980600\n",
      "\n",
      "step 12500, lr: 0.000358, entropy loss: 0.029651, l2 loss: 877.993835, total loss: 0.091111\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.980100\n",
      "\n",
      "step 12600, lr: 0.000358, entropy loss: 0.010298, l2 loss: 877.661316, total loss: 0.071734\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980600\n",
      "\n",
      "step 12700, lr: 0.000341, entropy loss: 0.041169, l2 loss: 875.900269, total loss: 0.102482\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.979400\n",
      "\n",
      "step 12800, lr: 0.000341, entropy loss: 0.023688, l2 loss: 875.980591, total loss: 0.085006\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980200\n",
      "\n",
      "step 12900, lr: 0.000341, entropy loss: 0.053483, l2 loss: 875.257629, total loss: 0.114751\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.980500\n",
      "\n",
      "step 13000, lr: 0.000341, entropy loss: 0.021411, l2 loss: 875.498657, total loss: 0.082696\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981100\n",
      "\n",
      "step 13100, lr: 0.000341, entropy loss: 0.025175, l2 loss: 874.921570, total loss: 0.086419\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.979500\n",
      "\n",
      "step 13200, lr: 0.000341, entropy loss: 0.024368, l2 loss: 874.276855, total loss: 0.085567\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.979900\n",
      "\n",
      "step 13300, lr: 0.000324, entropy loss: 0.011727, l2 loss: 874.497681, total loss: 0.072942\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981000\n",
      "\n",
      "step 13400, lr: 0.000324, entropy loss: 0.011227, l2 loss: 874.371460, total loss: 0.072433\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981700\n",
      "\n",
      "step 13500, lr: 0.000324, entropy loss: 0.005212, l2 loss: 873.392090, total loss: 0.066349\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.979300\n",
      "\n",
      "step 13600, lr: 0.000324, entropy loss: 0.048027, l2 loss: 872.443542, total loss: 0.109098\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.980400\n",
      "\n",
      "step 13700, lr: 0.000324, entropy loss: 0.061179, l2 loss: 872.313843, total loss: 0.122241\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.979800\n",
      "\n",
      "step 13800, lr: 0.000324, entropy loss: 0.046991, l2 loss: 871.886169, total loss: 0.108023\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.979900\n",
      "\n",
      "step 13900, lr: 0.000307, entropy loss: 0.014889, l2 loss: 871.089478, total loss: 0.075865\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980900\n",
      "\n",
      "step 14000, lr: 0.000307, entropy loss: 0.015932, l2 loss: 870.642334, total loss: 0.076877\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981900\n",
      "\n",
      "step 14100, lr: 0.000307, entropy loss: 0.026781, l2 loss: 870.136841, total loss: 0.087690\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980800\n",
      "\n",
      "step 14200, lr: 0.000307, entropy loss: 0.041002, l2 loss: 870.127258, total loss: 0.101911\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981800\n",
      "\n",
      "step 14300, lr: 0.000307, entropy loss: 0.042011, l2 loss: 869.513245, total loss: 0.102877\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.980600\n",
      "\n",
      "step 14400, lr: 0.000307, entropy loss: 0.030967, l2 loss: 869.437683, total loss: 0.091828\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.979200\n",
      "\n",
      "step 14500, lr: 0.000292, entropy loss: 0.011547, l2 loss: 869.125610, total loss: 0.072386\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980800\n",
      "\n",
      "step 14600, lr: 0.000292, entropy loss: 0.026291, l2 loss: 868.502686, total loss: 0.087086\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.981300\n",
      "\n",
      "step 14700, lr: 0.000292, entropy loss: 0.034655, l2 loss: 867.817871, total loss: 0.095402\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980700\n",
      "\n",
      "step 14800, lr: 0.000292, entropy loss: 0.012105, l2 loss: 867.515320, total loss: 0.072831\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980600\n",
      "\n",
      "step 14900, lr: 0.000292, entropy loss: 0.016246, l2 loss: 866.694702, total loss: 0.076914\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.978600\n",
      "\n",
      "step 15000, lr: 0.000292, entropy loss: 0.039285, l2 loss: 866.226929, total loss: 0.099921\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980900\n",
      "\n",
      "step 15100, lr: 0.000277, entropy loss: 0.022808, l2 loss: 865.355347, total loss: 0.083383\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980900\n",
      "\n",
      "step 15200, lr: 0.000277, entropy loss: 0.028412, l2 loss: 864.844177, total loss: 0.088951\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.980200\n",
      "\n",
      "step 15300, lr: 0.000277, entropy loss: 0.018839, l2 loss: 864.757996, total loss: 0.079372\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980100\n",
      "\n",
      "step 15400, lr: 0.000277, entropy loss: 0.028506, l2 loss: 864.577820, total loss: 0.089026\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980600\n",
      "\n",
      "step 15500, lr: 0.000277, entropy loss: 0.040303, l2 loss: 864.309692, total loss: 0.100805\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.981600\n",
      "\n",
      "step 15600, lr: 0.000277, entropy loss: 0.034482, l2 loss: 863.704834, total loss: 0.094941\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.981100\n",
      "\n",
      "step 15700, lr: 0.000264, entropy loss: 0.067995, l2 loss: 863.337585, total loss: 0.128429\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.980900\n",
      "\n",
      "step 15800, lr: 0.000264, entropy loss: 0.028056, l2 loss: 862.972961, total loss: 0.088464\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980700\n",
      "\n",
      "step 15900, lr: 0.000264, entropy loss: 0.029386, l2 loss: 862.756897, total loss: 0.089779\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.979600\n",
      "\n",
      "step 16000, lr: 0.000264, entropy loss: 0.121903, l2 loss: 862.703125, total loss: 0.182292\n",
      "train accuracy: 0.970000\n",
      "test accuracy: 0.979500\n",
      "\n",
      "step 16100, lr: 0.000264, entropy loss: 0.032899, l2 loss: 862.390381, total loss: 0.093266\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981600\n",
      "\n",
      "step 16200, lr: 0.000264, entropy loss: 0.020068, l2 loss: 861.657776, total loss: 0.080384\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981600\n",
      "\n",
      "step 16300, lr: 0.000250, entropy loss: 0.022194, l2 loss: 861.427673, total loss: 0.082494\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.979300\n",
      "\n",
      "step 16400, lr: 0.000250, entropy loss: 0.012390, l2 loss: 860.631592, total loss: 0.072635\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980700\n",
      "\n",
      "step 16500, lr: 0.000250, entropy loss: 0.024090, l2 loss: 860.577087, total loss: 0.084331\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.981000\n",
      "\n",
      "step 16600, lr: 0.000250, entropy loss: 0.012212, l2 loss: 860.541260, total loss: 0.072450\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981800\n",
      "\n",
      "step 16700, lr: 0.000250, entropy loss: 0.032825, l2 loss: 860.548096, total loss: 0.093063\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.981300\n",
      "\n",
      "step 16800, lr: 0.000250, entropy loss: 0.020062, l2 loss: 860.112610, total loss: 0.080269\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981800\n",
      "\n",
      "step 16900, lr: 0.000238, entropy loss: 0.024225, l2 loss: 859.299683, total loss: 0.084376\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981600\n",
      "\n",
      "step 17000, lr: 0.000238, entropy loss: 0.016725, l2 loss: 858.736694, total loss: 0.076837\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981200\n",
      "\n",
      "step 17100, lr: 0.000238, entropy loss: 0.057187, l2 loss: 858.378662, total loss: 0.117274\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.980800\n",
      "\n",
      "step 17200, lr: 0.000238, entropy loss: 0.017107, l2 loss: 857.894836, total loss: 0.077159\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981200\n",
      "\n",
      "step 17300, lr: 0.000238, entropy loss: 0.030612, l2 loss: 857.429688, total loss: 0.090632\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980300\n",
      "\n",
      "step 17400, lr: 0.000238, entropy loss: 0.013098, l2 loss: 856.731812, total loss: 0.073069\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980900\n",
      "\n",
      "step 17500, lr: 0.000226, entropy loss: 0.030064, l2 loss: 856.262939, total loss: 0.090003\n",
      "train accuracy: 1.000000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test accuracy: 0.982000\n",
      "\n",
      "step 17600, lr: 0.000226, entropy loss: 0.032980, l2 loss: 856.063171, total loss: 0.092905\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.980900\n",
      "\n",
      "step 17700, lr: 0.000226, entropy loss: 0.015165, l2 loss: 855.885132, total loss: 0.075077\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981300\n",
      "\n",
      "step 17800, lr: 0.000226, entropy loss: 0.017485, l2 loss: 854.874695, total loss: 0.077326\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981500\n",
      "\n",
      "step 17900, lr: 0.000226, entropy loss: 0.018111, l2 loss: 854.586365, total loss: 0.077932\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980500\n",
      "\n",
      "step 18000, lr: 0.000226, entropy loss: 0.019991, l2 loss: 854.245728, total loss: 0.079789\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981500\n",
      "\n",
      "step 18100, lr: 0.000215, entropy loss: 0.026767, l2 loss: 853.873657, total loss: 0.086538\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980700\n",
      "\n",
      "step 18200, lr: 0.000215, entropy loss: 0.024014, l2 loss: 853.272034, total loss: 0.083743\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980800\n",
      "\n",
      "step 18300, lr: 0.000215, entropy loss: 0.019111, l2 loss: 853.832275, total loss: 0.078880\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981100\n",
      "\n",
      "step 18400, lr: 0.000215, entropy loss: 0.017253, l2 loss: 853.120483, total loss: 0.076971\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.981400\n",
      "\n",
      "step 18500, lr: 0.000215, entropy loss: 0.012498, l2 loss: 852.368958, total loss: 0.072164\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.979100\n",
      "\n",
      "step 18600, lr: 0.000215, entropy loss: 0.028172, l2 loss: 852.603882, total loss: 0.087854\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.981400\n",
      "\n",
      "step 18700, lr: 0.000204, entropy loss: 0.023701, l2 loss: 852.114380, total loss: 0.083349\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980700\n",
      "\n",
      "step 18800, lr: 0.000204, entropy loss: 0.014066, l2 loss: 852.299683, total loss: 0.073727\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980300\n",
      "\n",
      "step 18900, lr: 0.000204, entropy loss: 0.020559, l2 loss: 851.985718, total loss: 0.080198\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.982000\n",
      "\n",
      "step 19000, lr: 0.000204, entropy loss: 0.048582, l2 loss: 851.625854, total loss: 0.108195\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.981000\n",
      "\n",
      "step 19100, lr: 0.000204, entropy loss: 0.045416, l2 loss: 851.193481, total loss: 0.104999\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.980100\n",
      "\n",
      "step 19200, lr: 0.000204, entropy loss: 0.021838, l2 loss: 850.996338, total loss: 0.081408\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981800\n",
      "\n",
      "step 19300, lr: 0.000194, entropy loss: 0.049631, l2 loss: 850.561462, total loss: 0.109171\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.981700\n",
      "\n",
      "step 19400, lr: 0.000194, entropy loss: 0.023008, l2 loss: 850.550842, total loss: 0.082547\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980400\n",
      "\n",
      "step 19500, lr: 0.000194, entropy loss: 0.034084, l2 loss: 850.363403, total loss: 0.093610\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.980600\n",
      "\n",
      "step 19600, lr: 0.000194, entropy loss: 0.074591, l2 loss: 849.694763, total loss: 0.134070\n",
      "train accuracy: 0.980000\n",
      "test accuracy: 0.982000\n",
      "\n",
      "step 19700, lr: 0.000194, entropy loss: 0.017400, l2 loss: 849.017883, total loss: 0.076831\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981300\n",
      "\n",
      "step 19800, lr: 0.000194, entropy loss: 0.014044, l2 loss: 848.876831, total loss: 0.073466\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981500\n",
      "\n",
      "step 19900, lr: 0.000184, entropy loss: 0.026139, l2 loss: 848.729187, total loss: 0.085550\n",
      "train accuracy: 0.990000\n",
      "test accuracy: 0.980000\n",
      "\n",
      "step 20000, lr: 0.000184, entropy loss: 0.018749, l2 loss: 848.770386, total loss: 0.078163\n",
      "train accuracy: 1.000000\n",
      "test accuracy: 0.981300\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()\n",
    "\n",
    "# Train\n",
    "for step in range(20000):\n",
    "    #if step < 100:\n",
    "    #    lr = 0.05\n",
    "    #elif step < 1400:\n",
    "    #    lr = 0.01\n",
    "    #elif step < 2400:\n",
    "    #    lr = 0.001\n",
    "    #elif step < 3200:\n",
    "    #    lr = 0.0001\n",
    "    #else:\n",
    "    #    lr = 0.00001\n",
    "    lr = 0.001 * (0.95 ** (step//600) )\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    _, loss, l2_loss_value, total_loss_value = sess.run([train_step, cross_entropy, l2_loss, total_loss], \n",
    "             feed_dict={x: batch_xs, y_: batch_ys, learning_rate: lr})\n",
    "    if (step+1) % 100 == 0:\n",
    "        print('step %d, lr: %f, entropy loss: %f, l2 loss: %f, total loss: %f'%(step+1, lr, loss, l2_loss_value, total_loss_value))\n",
    "        print('train accuracy: %f'%(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys})) )\n",
    "        print('test accuracy: %f'%(sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})) )\n",
    "        print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
